import sys
from scrapy_redis.spiders import RedisCrawlSpider
sys.path.append("..")
from redisoper import CRedis

import scrapy
import re
import os
import  urlparse
from items import FruitImgItem
from scrapy import Spider
from scrapy.http import Request


class BdImgSplider(RedisCrawlSpider):
    redis_key = 'bd_spider:start_urls';
    name = "bdimgs"

    def make_requests_from_url(self, url):
        query = urlparse.urlparse(url).query
        qs_dict = dict(urlparse.parse_qsl(query))
        filename = qs_dict['filename']
        request = Request(url, dont_filter=True)
        request.meta['filename'] = filename
        return request

    def __init__(self):
        self.r = CRedis()


    def parse(self, response):
        html=response.body
        reg = r'"objURL":"(.*?)"'
        imgre=re.compile(reg)
        print imgre
        imglist=re.findall(imgre,html);
        items = []
        for imgurl in imglist:
            item = FruitImgItem()
            item['id'] = response.meta['filename']
            item['image_url']=imgurl;
            items.append(item)
        return items

